{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import json\n",
    "\n",
    "# Load URLs from the JSON file\n",
    "with open('feature-extraction_downloads.json', 'r') as json_file:\n",
    "    data = json.load(json_file)\n",
    "    # urls = data.get('urls', [])\n",
    "ids = [i[\"id\"] for i in data[\"models\"]]\n",
    "urls = [f\"https://huggingface.co/{i}/tree/main/onnx\" for i in ids]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'author': 'TaylorAI',\n",
       " 'authorData': {'avatarUrl': 'https://aeiljuispo.cloudimg.io/v7/https://cdn-uploads.huggingface.co/production/uploads/63917e16b6b839bb61483dbf/Utq89ebo7Glxfls0QZnxK.png?w=200&h=200&f=face',\n",
       "  'fullname': 'Taylor',\n",
       "  'name': 'TaylorAI',\n",
       "  'type': 'org',\n",
       "  'isHf': False},\n",
       " 'downloads': 1752,\n",
       " 'gated': False,\n",
       " 'id': 'TaylorAI/gte-tiny',\n",
       " 'lastModified': '2023-10-07T05:20:49.000Z',\n",
       " 'likes': 102,\n",
       " 'pipeline_tag': 'sentence-similarity',\n",
       " 'private': False,\n",
       " 'repoType': 'model',\n",
       " 'isLikedByUser': False}"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[\"models\"][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://huggingface.co/TaylorAI/gte-tiny/tree/main/onnx | 22.9\n",
      "https://huggingface.co/Supabase/gte-small/tree/main/onnx | 34\n",
      "https://huggingface.co/Xenova/all-MiniLM-L6-v2/tree/main/onnx | 23\n",
      "https://huggingface.co/Xenova/bge-large-en-v1.5/tree/main/onnx | 337\n",
      "https://huggingface.co/Supabase/bge-small-en/tree/main/onnx | 34\n",
      "https://huggingface.co/Xenova/gte-small/tree/main/onnx | 34\n",
      "https://huggingface.co/Xenova/all-mpnet-base-v2/tree/main/onnx | 110\n",
      "https://huggingface.co/Xenova/paraphrase-mpnet-base-v2/tree/main/onnx | 110\n",
      "https://huggingface.co/Xenova/all-MiniLM-L12-v2/tree/main/onnx | 34\n",
      "https://huggingface.co/Xenova/multilingual-e5-small/tree/main/onnx | 118\n",
      "https://huggingface.co/Xenova/gte-large/tree/main/onnx | 337\n",
      "https://huggingface.co/Xenova/bge-base-en-v1.5/tree/main/onnx | 110\n",
      "https://huggingface.co/Xenova/all-roberta-large-v1/tree/main/onnx | 357\n",
      "https://huggingface.co/Xenova/distiluse-base-multilingual-cased-v2/tree/main/onnx | 135\n",
      "https://huggingface.co/Xenova/paraphrase-multilingual-mpnet-base-v2/tree/main/onnx | 279\n",
      "https://huggingface.co/Xenova/bge-large-zh/tree/main/onnx | 327\n",
      "https://huggingface.co/Xenova/multilingual-e5-base/tree/main/onnx | 279\n",
      "https://huggingface.co/Xenova/bge-small-en-v1.5/tree/main/onnx | 34\n",
      "https://huggingface.co/Xenova/paraphrase-albert-small-v2/tree/main/onnx | 39.7\n",
      "https://huggingface.co/Xenova/paraphrase-albert-base-v2/tree/main/onnx | 40\n",
      "https://huggingface.co/Xenova/squeezebert-uncased/tree/main/onnx | 51.2\n",
      "https://huggingface.co/Xenova/squeezebert-mnli/tree/main/onnx | 51.3\n",
      "https://huggingface.co/Xenova/vit-base-patch16-224-in21k/tree/main/onnx | 87.5\n",
      "https://huggingface.co/Xenova/all-distilroberta-v1/tree/main/onnx | 82.1\n",
      "https://huggingface.co/Xenova/paraphrase-multilingual-MiniLM-L12-v2/tree/main/onnx | 118\n",
      "https://huggingface.co/Xenova/paraphrase-MiniLM-L6-v2/tree/main/onnx | 23\n",
      "https://huggingface.co/Xenova/bert-base-nli-mean-tokens/tree/main/onnx | 110\n",
      "https://huggingface.co/Xenova/distilbert-base-nli-mean-tokens/tree/main/onnx | 66.9\n",
      "https://huggingface.co/Xenova/distilbert-base-nli-stsb-mean-tokens/tree/main/onnx | 66.9\n",
      "https://huggingface.co/Xenova/distiluse-base-multilingual-cased-v1/tree/main/onnx | 135\n",
      "https://huggingface.co/Xenova/msmarco-distilbert-base-v4/tree/main/onnx | 66.9\n",
      "https://huggingface.co/Xenova/multi-qa-MiniLM-L6-cos-v1/tree/main/onnx | 23\n",
      "https://huggingface.co/Xenova/multi-qa-distilbert-cos-v1/tree/main/onnx | 66.9\n",
      "https://huggingface.co/Xenova/multi-qa-mpnet-base-cos-v1/tree/main/onnx | 110\n",
      "https://huggingface.co/Xenova/multi-qa-mpnet-base-dot-v1/tree/main/onnx | 110\n",
      "https://huggingface.co/Xenova/nli-mpnet-base-v2/tree/main/onnx | 110\n",
      "https://huggingface.co/Xenova/paraphrase-MiniLM-L3-v2/tree/main/onnx | 17.5\n",
      "https://huggingface.co/Xenova/xlm-r-100langs-bert-base-nli-stsb-mean-tokens/tree/main/onnx | 279\n",
      "https://huggingface.co/Xenova/dino-vitb16/tree/main/onnx | 87.5\n",
      "https://huggingface.co/Xenova/dino-vits8/tree/main/onnx | 23.4\n",
      "https://huggingface.co/Xenova/dino-vitb8/tree/main/onnx | 88.8\n",
      "https://huggingface.co/Xenova/dino-vits16/tree/main/onnx | 22.7\n",
      "https://huggingface.co/Xenova/scibert_scivocab_uncased/tree/main/onnx | 111\n",
      "https://huggingface.co/Xenova/spanbert-large-cased/tree/main/onnx | 335\n",
      "https://huggingface.co/Xenova/spanbert-base-cased/tree/main/onnx | 109\n",
      "https://huggingface.co/sdan/simple-embeddings/tree/main/onnx | 23\n",
      "https://huggingface.co/Xenova/sentence_bert/tree/main/onnx | 110\n",
      "https://huggingface.co/Xenova/e5-small-v2/tree/main/onnx | 34\n",
      "https://huggingface.co/Xenova/SapBERT-from-PubMedBERT-fulltext/tree/main/onnx | 110\n",
      "https://huggingface.co/Xenova/indobert-base-p1/tree/main/onnx | 125\n",
      "https://huggingface.co/Xenova/UMLSBert_ENG/tree/main/onnx | 110\n",
      "https://huggingface.co/Xenova/rubert-base-cased/tree/main/onnx | 178\n",
      "https://huggingface.co/Xenova/kobert/tree/main/onnx | 92.8\n",
      "https://huggingface.co/Xenova/e5-small/tree/main/onnx | 34\n",
      "https://huggingface.co/Xenova/e5-large/tree/main/onnx | 337\n",
      "https://huggingface.co/Xenova/e5-large-v2/tree/main/onnx | 337\n",
      "https://huggingface.co/Xenova/e5-base/tree/main/onnx | 110\n",
      "https://huggingface.co/Xenova/e5-base-v2/tree/main/onnx | 110\n",
      "https://huggingface.co/Xenova/instructor-base/tree/main/onnx | 110\n",
      "https://huggingface.co/Xenova/instructor-large/tree/main/onnx | 337\n",
      "https://huggingface.co/Xenova/sentence-t5-large/tree/main/onnx | 337\n",
      "https://huggingface.co/Xenova/multilingual-e5-large/tree/main/onnx | 562\n",
      "https://huggingface.co/Xenova/mms-300m/tree/main/onnx | 318\n",
      "https://huggingface.co/Xenova/mms-1b/tree/main/onnx | 969\n",
      "https://huggingface.co/Supabase/e5-small-v2/tree/main/onnx | 34\n",
      "https://huggingface.co/Supabase/all-MiniLM-L6-v2/tree/main/onnx | 23\n",
      "https://huggingface.co/Xenova/gte-base/tree/main/onnx | 110\n",
      "https://huggingface.co/Xenova/bge-small-en/tree/main/onnx | 34\n",
      "https://huggingface.co/Xenova/bge-base-en/tree/main/onnx | 110\n",
      "https://huggingface.co/Xenova/bge-large-en/tree/main/onnx | 337\n",
      "https://huggingface.co/ggrn/bge-small-en/tree/main/onnx | 34\n",
      "https://huggingface.co/Xenova/bge-base-zh/tree/main/onnx | 103\n",
      "https://huggingface.co/Xenova/bge-large-zh-noinstruct/tree/main/onnx | 327\n",
      "https://huggingface.co/Xenova/bge-small-zh/tree/main/onnx | 24\n",
      "https://huggingface.co/Xenova/ClinicalBERT/tree/main/onnx | 229\n",
      "https://huggingface.co/Xenova/LaBSE/tree/main/onnx | 472\n",
      "https://huggingface.co/Xenova/wavlm-base/tree/main/onnx | 95.8\n",
      "https://huggingface.co/Xenova/wavlm-base-plus/tree/main/onnx | 95.8\n",
      "https://huggingface.co/Xenova/wavlm-large/tree/main/onnx | 319\n",
      "https://huggingface.co/Xenova/sentence-camembert-large/tree/main/onnx | 339\n",
      "https://huggingface.co/Xenova/herbert-base-cased/tree/main/onnx | 125\n",
      "https://huggingface.co/Xenova/herbert-large-cased/tree/main/onnx | 357\n",
      "https://huggingface.co/Xenova/bge-large-zh-v1.5/tree/main/onnx | 327\n",
      "https://huggingface.co/Xenova/bge-base-zh-v1.5/tree/main/onnx | 103\n",
      "https://huggingface.co/Xenova/bge-small-zh-v1.5/tree/main/onnx | 24\n",
      "https://huggingface.co/leolee9086/text2vec-base-chinese/tree/main/onnx | 103\n",
      "https://huggingface.co/Xenova/long-t5-encodec-tglobal-base/tree/main/onnx | 291\n"
     ]
    }
   ],
   "source": [
    "def extract_size_from_url(url):\n",
    "    try:\n",
    "        response = requests.get(url)\n",
    "        if response.status_code == 200:\n",
    "            soup = BeautifulSoup(response.text, 'html.parser')\n",
    "            \n",
    "            # Find the 'a' tag with the specified title attribute\n",
    "            a_tag = soup.find('a', title=\"Download file\")\n",
    "            if a_tag:\n",
    "                size = a_tag.text.strip()  # Extract the size text\n",
    "                return size\n",
    "            else:\n",
    "                return \"Size not found\"\n",
    "        else:\n",
    "            return \"Failed to retrieve the page\"\n",
    "    except requests.exceptions.RequestException as e:\n",
    "        return f\"Request error: {e}\"\n",
    "\n",
    "# Iterate through the URLs and extract values\n",
    "sizes = []\n",
    "\n",
    "for url in urls:\n",
    "    values = extract_values_from_url(url)[-1].split(\" MB\")[0]\n",
    "    sizes.append([url.split(\"https://huggingface.co/\")[1].split(\"/tree/main/onnx\")[0],values])\n",
    "    print(f\"{url} | {values}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['TaylorAI/gte-tiny', '22.9'],\n",
       " ['Supabase/gte-small', '34'],\n",
       " ['Xenova/all-MiniLM-L6-v2', '23'],\n",
       " ['Xenova/bge-large-en-v1.5', '337'],\n",
       " ['Supabase/bge-small-en', '34'],\n",
       " ['Xenova/gte-small', '34'],\n",
       " ['Xenova/all-mpnet-base-v2', '110'],\n",
       " ['Xenova/paraphrase-mpnet-base-v2', '110'],\n",
       " ['Xenova/all-MiniLM-L12-v2', '34'],\n",
       " ['Xenova/multilingual-e5-small', '118'],\n",
       " ['Xenova/gte-large', '337'],\n",
       " ['Xenova/bge-base-en-v1.5', '110'],\n",
       " ['Xenova/all-roberta-large-v1', '357'],\n",
       " ['Xenova/distiluse-base-multilingual-cased-v2', '135'],\n",
       " ['Xenova/paraphrase-multilingual-mpnet-base-v2', '279'],\n",
       " ['Xenova/bge-large-zh', '327'],\n",
       " ['Xenova/multilingual-e5-base', '279'],\n",
       " ['Xenova/bge-small-en-v1.5', '34'],\n",
       " ['Xenova/paraphrase-albert-small-v2', '39.7'],\n",
       " ['Xenova/paraphrase-albert-base-v2', '40'],\n",
       " ['Xenova/squeezebert-uncased', '51.2'],\n",
       " ['Xenova/squeezebert-mnli', '51.3'],\n",
       " ['Xenova/vit-base-patch16-224-in21k', '87.5'],\n",
       " ['Xenova/all-distilroberta-v1', '82.1'],\n",
       " ['Xenova/paraphrase-multilingual-MiniLM-L12-v2', '118'],\n",
       " ['Xenova/paraphrase-MiniLM-L6-v2', '23'],\n",
       " ['Xenova/bert-base-nli-mean-tokens', '110'],\n",
       " ['Xenova/distilbert-base-nli-mean-tokens', '66.9'],\n",
       " ['Xenova/distilbert-base-nli-stsb-mean-tokens', '66.9'],\n",
       " ['Xenova/distiluse-base-multilingual-cased-v1', '135'],\n",
       " ['Xenova/msmarco-distilbert-base-v4', '66.9'],\n",
       " ['Xenova/multi-qa-MiniLM-L6-cos-v1', '23'],\n",
       " ['Xenova/multi-qa-distilbert-cos-v1', '66.9'],\n",
       " ['Xenova/multi-qa-mpnet-base-cos-v1', '110'],\n",
       " ['Xenova/multi-qa-mpnet-base-dot-v1', '110'],\n",
       " ['Xenova/nli-mpnet-base-v2', '110'],\n",
       " ['Xenova/paraphrase-MiniLM-L3-v2', '17.5'],\n",
       " ['Xenova/xlm-r-100langs-bert-base-nli-stsb-mean-tokens', '279'],\n",
       " ['Xenova/dino-vitb16', '87.5'],\n",
       " ['Xenova/dino-vits8', '23.4'],\n",
       " ['Xenova/dino-vitb8', '88.8'],\n",
       " ['Xenova/dino-vits16', '22.7'],\n",
       " ['Xenova/scibert_scivocab_uncased', '111'],\n",
       " ['Xenova/spanbert-large-cased', '335'],\n",
       " ['Xenova/spanbert-base-cased', '109'],\n",
       " ['sdan/simple-embeddings', '23'],\n",
       " ['Xenova/sentence_bert', '110'],\n",
       " ['Xenova/e5-small-v2', '34'],\n",
       " ['Xenova/SapBERT-from-PubMedBERT-fulltext', '110'],\n",
       " ['Xenova/indobert-base-p1', '125'],\n",
       " ['Xenova/UMLSBert_ENG', '110'],\n",
       " ['Xenova/rubert-base-cased', '178'],\n",
       " ['Xenova/kobert', '92.8'],\n",
       " ['Xenova/e5-small', '34'],\n",
       " ['Xenova/e5-large', '337'],\n",
       " ['Xenova/e5-large-v2', '337'],\n",
       " ['Xenova/e5-base', '110'],\n",
       " ['Xenova/e5-base-v2', '110'],\n",
       " ['Xenova/instructor-base', '110'],\n",
       " ['Xenova/instructor-large', '337'],\n",
       " ['Xenova/sentence-t5-large', '337'],\n",
       " ['Xenova/multilingual-e5-large', '562'],\n",
       " ['Xenova/mms-300m', '318'],\n",
       " ['Xenova/mms-1b', '969'],\n",
       " ['Supabase/e5-small-v2', '34'],\n",
       " ['Supabase/all-MiniLM-L6-v2', '23'],\n",
       " ['Xenova/gte-base', '110'],\n",
       " ['Xenova/bge-small-en', '34'],\n",
       " ['Xenova/bge-base-en', '110'],\n",
       " ['Xenova/bge-large-en', '337'],\n",
       " ['ggrn/bge-small-en', '34'],\n",
       " ['Xenova/bge-base-zh', '103'],\n",
       " ['Xenova/bge-large-zh-noinstruct', '327'],\n",
       " ['Xenova/bge-small-zh', '24'],\n",
       " ['Xenova/ClinicalBERT', '229'],\n",
       " ['Xenova/LaBSE', '472'],\n",
       " ['Xenova/wavlm-base', '95.8'],\n",
       " ['Xenova/wavlm-base-plus', '95.8'],\n",
       " ['Xenova/wavlm-large', '319'],\n",
       " ['Xenova/sentence-camembert-large', '339'],\n",
       " ['Xenova/herbert-base-cased', '125'],\n",
       " ['Xenova/herbert-large-cased', '357'],\n",
       " ['Xenova/bge-large-zh-v1.5', '327'],\n",
       " ['Xenova/bge-base-zh-v1.5', '103'],\n",
       " ['Xenova/bge-small-zh-v1.5', '24'],\n",
       " ['leolee9086/text2vec-base-chinese', '103'],\n",
       " ['Xenova/long-t5-encodec-tglobal-base', '291']]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sizes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "sizes_backup = [['TaylorAI/gte-tiny', '22.9'],\n",
    " ['Supabase/gte-small', '34'],\n",
    " ['Xenova/all-MiniLM-L6-v2', '23'],\n",
    " ['Xenova/bge-large-en-v1.5', '337'],\n",
    " ['Supabase/bge-small-en', '34'],\n",
    " ['Xenova/gte-small', '34'],\n",
    " ['Xenova/all-mpnet-base-v2', '110'],\n",
    " ['Xenova/paraphrase-mpnet-base-v2', '110'],\n",
    " ['Xenova/all-MiniLM-L12-v2', '34'],\n",
    " ['Xenova/multilingual-e5-small', '118'],\n",
    " ['Xenova/gte-large', '337'],\n",
    " ['Xenova/bge-base-en-v1.5', '110'],\n",
    " ['Xenova/all-roberta-large-v1', '357'],\n",
    " ['Xenova/distiluse-base-multilingual-cased-v2', '135'],\n",
    " ['Xenova/paraphrase-multilingual-mpnet-base-v2', '279'],\n",
    " ['Xenova/bge-large-zh', '327'],\n",
    " ['Xenova/multilingual-e5-base', '279'],\n",
    " ['Xenova/bge-small-en-v1.5', '34'],\n",
    " ['Xenova/paraphrase-albert-small-v2', '39.7'],\n",
    " ['Xenova/paraphrase-albert-base-v2', '40'],\n",
    " ['Xenova/squeezebert-uncased', '51.2'],\n",
    " ['Xenova/squeezebert-mnli', '51.3'],\n",
    " ['Xenova/vit-base-patch16-224-in21k', '87.5'],\n",
    " ['Xenova/all-distilroberta-v1', '82.1'],\n",
    " ['Xenova/paraphrase-multilingual-MiniLM-L12-v2', '118'],\n",
    " ['Xenova/paraphrase-MiniLM-L6-v2', '23'],\n",
    " ['Xenova/bert-base-nli-mean-tokens', '110'],\n",
    " ['Xenova/distilbert-base-nli-mean-tokens', '66.9'],\n",
    " ['Xenova/distilbert-base-nli-stsb-mean-tokens', '66.9'],\n",
    " ['Xenova/distiluse-base-multilingual-cased-v1', '135'],\n",
    " ['Xenova/msmarco-distilbert-base-v4', '66.9'],\n",
    " ['Xenova/multi-qa-MiniLM-L6-cos-v1', '23'],\n",
    " ['Xenova/multi-qa-distilbert-cos-v1', '66.9'],\n",
    " ['Xenova/multi-qa-mpnet-base-cos-v1', '110'],\n",
    " ['Xenova/multi-qa-mpnet-base-dot-v1', '110'],\n",
    " ['Xenova/nli-mpnet-base-v2', '110'],\n",
    " ['Xenova/paraphrase-MiniLM-L3-v2', '17.5'],\n",
    " ['Xenova/xlm-r-100langs-bert-base-nli-stsb-mean-tokens', '279'],\n",
    " ['Xenova/dino-vitb16', '87.5'],\n",
    " ['Xenova/dino-vits8', '23.4'],\n",
    " ['Xenova/dino-vitb8', '88.8'],\n",
    " ['Xenova/dino-vits16', '22.7'],\n",
    " ['Xenova/scibert_scivocab_uncased', '111'],\n",
    " ['Xenova/spanbert-large-cased', '335'],\n",
    " ['Xenova/spanbert-base-cased', '109'],\n",
    " ['sdan/simple-embeddings', '23'],\n",
    " ['Xenova/sentence_bert', '110'],\n",
    " ['Xenova/e5-small-v2', '34'],\n",
    " ['Xenova/SapBERT-from-PubMedBERT-fulltext', '110'],\n",
    " ['Xenova/indobert-base-p1', '125'],\n",
    " ['Xenova/UMLSBert_ENG', '110'],\n",
    " ['Xenova/rubert-base-cased', '178'],\n",
    " ['Xenova/kobert', '92.8'],\n",
    " ['Xenova/e5-small', '34'],\n",
    " ['Xenova/e5-large', '337'],\n",
    " ['Xenova/e5-large-v2', '337'],\n",
    " ['Xenova/e5-base', '110'],\n",
    " ['Xenova/e5-base-v2', '110'],\n",
    " ['Xenova/instructor-base', '110'],\n",
    " ['Xenova/instructor-large', '337'],\n",
    " ['Xenova/sentence-t5-large', '337'],\n",
    " ['Xenova/multilingual-e5-large', '562'],\n",
    " ['Xenova/mms-300m', '318'],\n",
    " ['Xenova/mms-1b', '969'],\n",
    " ['Supabase/e5-small-v2', '34'],\n",
    " ['Supabase/all-MiniLM-L6-v2', '23'],\n",
    " ['Xenova/gte-base', '110'],\n",
    " ['Xenova/bge-small-en', '34'],\n",
    " ['Xenova/bge-base-en', '110'],\n",
    " ['Xenova/bge-large-en', '337'],\n",
    " ['ggrn/bge-small-en', '34'],\n",
    " ['Xenova/bge-base-zh', '103'],\n",
    " ['Xenova/bge-large-zh-noinstruct', '327'],\n",
    " ['Xenova/bge-small-zh', '24'],\n",
    " ['Xenova/ClinicalBERT', '229'],\n",
    " ['Xenova/LaBSE', '472'],\n",
    " ['Xenova/wavlm-base', '95.8'],\n",
    " ['Xenova/wavlm-base-plus', '95.8'],\n",
    " ['Xenova/wavlm-large', '319'],\n",
    " ['Xenova/sentence-camembert-large', '339'],\n",
    " ['Xenova/herbert-base-cased', '125'],\n",
    " ['Xenova/herbert-large-cased', '357'],\n",
    " ['Xenova/bge-large-zh-v1.5', '327'],\n",
    " ['Xenova/bge-base-zh-v1.5', '103'],\n",
    " ['Xenova/bge-small-zh-v1.5', '24'],\n",
    " ['leolee9086/text2vec-base-chinese', '103'],\n",
    " ['Xenova/long-t5-encodec-tglobal-base', '291']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'TaylorAI/gte-tiny': '22.9',\n",
       " 'Supabase/gte-small': '34',\n",
       " 'Xenova/all-MiniLM-L6-v2': '23',\n",
       " 'Xenova/bge-large-en-v1.5': '337',\n",
       " 'Supabase/bge-small-en': '34',\n",
       " 'Xenova/gte-small': '34',\n",
       " 'Xenova/all-mpnet-base-v2': '110',\n",
       " 'Xenova/paraphrase-mpnet-base-v2': '110',\n",
       " 'Xenova/all-MiniLM-L12-v2': '34',\n",
       " 'Xenova/multilingual-e5-small': '118',\n",
       " 'Xenova/gte-large': '337',\n",
       " 'Xenova/bge-base-en-v1.5': '110',\n",
       " 'Xenova/all-roberta-large-v1': '357',\n",
       " 'Xenova/distiluse-base-multilingual-cased-v2': '135',\n",
       " 'Xenova/paraphrase-multilingual-mpnet-base-v2': '279',\n",
       " 'Xenova/bge-large-zh': '327',\n",
       " 'Xenova/multilingual-e5-base': '279',\n",
       " 'Xenova/bge-small-en-v1.5': '34',\n",
       " 'Xenova/paraphrase-albert-small-v2': '39.7',\n",
       " 'Xenova/paraphrase-albert-base-v2': '40',\n",
       " 'Xenova/squeezebert-uncased': '51.2',\n",
       " 'Xenova/squeezebert-mnli': '51.3',\n",
       " 'Xenova/vit-base-patch16-224-in21k': '87.5',\n",
       " 'Xenova/all-distilroberta-v1': '82.1',\n",
       " 'Xenova/paraphrase-multilingual-MiniLM-L12-v2': '118',\n",
       " 'Xenova/paraphrase-MiniLM-L6-v2': '23',\n",
       " 'Xenova/bert-base-nli-mean-tokens': '110',\n",
       " 'Xenova/distilbert-base-nli-mean-tokens': '66.9',\n",
       " 'Xenova/distilbert-base-nli-stsb-mean-tokens': '66.9',\n",
       " 'Xenova/distiluse-base-multilingual-cased-v1': '135',\n",
       " 'Xenova/msmarco-distilbert-base-v4': '66.9',\n",
       " 'Xenova/multi-qa-MiniLM-L6-cos-v1': '23',\n",
       " 'Xenova/multi-qa-distilbert-cos-v1': '66.9',\n",
       " 'Xenova/multi-qa-mpnet-base-cos-v1': '110',\n",
       " 'Xenova/multi-qa-mpnet-base-dot-v1': '110',\n",
       " 'Xenova/nli-mpnet-base-v2': '110',\n",
       " 'Xenova/paraphrase-MiniLM-L3-v2': '17.5',\n",
       " 'Xenova/xlm-r-100langs-bert-base-nli-stsb-mean-tokens': '279',\n",
       " 'Xenova/dino-vitb16': '87.5',\n",
       " 'Xenova/dino-vits8': '23.4',\n",
       " 'Xenova/dino-vitb8': '88.8',\n",
       " 'Xenova/dino-vits16': '22.7',\n",
       " 'Xenova/scibert_scivocab_uncased': '111',\n",
       " 'Xenova/spanbert-large-cased': '335',\n",
       " 'Xenova/spanbert-base-cased': '109',\n",
       " 'sdan/simple-embeddings': '23',\n",
       " 'Xenova/sentence_bert': '110',\n",
       " 'Xenova/e5-small-v2': '34',\n",
       " 'Xenova/SapBERT-from-PubMedBERT-fulltext': '110',\n",
       " 'Xenova/indobert-base-p1': '125',\n",
       " 'Xenova/UMLSBert_ENG': '110',\n",
       " 'Xenova/rubert-base-cased': '178',\n",
       " 'Xenova/kobert': '92.8',\n",
       " 'Xenova/e5-small': '34',\n",
       " 'Xenova/e5-large': '337',\n",
       " 'Xenova/e5-large-v2': '337',\n",
       " 'Xenova/e5-base': '110',\n",
       " 'Xenova/e5-base-v2': '110',\n",
       " 'Xenova/instructor-base': '110',\n",
       " 'Xenova/instructor-large': '337',\n",
       " 'Xenova/sentence-t5-large': '337',\n",
       " 'Xenova/multilingual-e5-large': '562',\n",
       " 'Xenova/mms-300m': '318',\n",
       " 'Xenova/mms-1b': '969',\n",
       " 'Supabase/e5-small-v2': '34',\n",
       " 'Supabase/all-MiniLM-L6-v2': '23',\n",
       " 'Xenova/gte-base': '110',\n",
       " 'Xenova/bge-small-en': '34',\n",
       " 'Xenova/bge-base-en': '110',\n",
       " 'Xenova/bge-large-en': '337',\n",
       " 'ggrn/bge-small-en': '34',\n",
       " 'Xenova/bge-base-zh': '103',\n",
       " 'Xenova/bge-large-zh-noinstruct': '327',\n",
       " 'Xenova/bge-small-zh': '24',\n",
       " 'Xenova/ClinicalBERT': '229',\n",
       " 'Xenova/LaBSE': '472',\n",
       " 'Xenova/wavlm-base': '95.8',\n",
       " 'Xenova/wavlm-base-plus': '95.8',\n",
       " 'Xenova/wavlm-large': '319',\n",
       " 'Xenova/sentence-camembert-large': '339',\n",
       " 'Xenova/herbert-base-cased': '125',\n",
       " 'Xenova/herbert-large-cased': '357',\n",
       " 'Xenova/bge-large-zh-v1.5': '327',\n",
       " 'Xenova/bge-base-zh-v1.5': '103',\n",
       " 'Xenova/bge-small-zh-v1.5': '24',\n",
       " 'leolee9086/text2vec-base-chinese': '103',\n",
       " 'Xenova/long-t5-encodec-tglobal-base': '291'}"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create a dictionary to easily look up sizes by id\n",
    "size_dict = dict(sizes)\n",
    "size_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "JSON updated and saved to 'your_output.json'\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "\n",
    "this_file = \"feature-extraction_trending\"\n",
    "\n",
    "with open(f\"{this_file}.json\", 'r') as json_file:\n",
    "    data = json.load(json_file)\n",
    "\n",
    "# Iterate over the \"models\" in your JSON\n",
    "for model in data[\"models\"]:\n",
    "    model_id = model[\"id\"]\n",
    "    if model_id in size_dict:\n",
    "        model[\"model_size\"] = size_dict[model_id]\n",
    "\n",
    "# Save the updated JSON to a file\n",
    "with open(f'{this_file}_sizes.json', 'w') as file:\n",
    "    json.dump(data, file, indent=4)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "py3.11",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
